package crawler.proxy;

import crawler.douban.DouBanSpider;
import org.apache.log4j.Logger;

import java.util.Set;

/**
 * USER: lintc 【lintiancong@zhuojianchina.com】
 * DATE: 2017-09-14 09:43
 */
public abstract class ProxySpider implements ProxySpiderService {

    protected static final Logger LOG = Logger.getLogger(ProxySpider.class);

    protected String catchUrl;

    protected String catchInvokeUrl;

    /**
     * 过滤代理类型（高匿、普匿、透明）
     */
    protected String filterStr = "匿";

    public ProxySpider(String catchUrl) {
        this.catchUrl = catchUrl;
    }

    public Set<Proxy> list(int index) {
        long start = System.currentTimeMillis();
        catchInvokeUrl = catchUrl.replace("index", index + "");
        Set<Proxy> list = getList(index);
        long timeCost = System.currentTimeMillis() - start;
        LOG.info("爬取代理IP (" + catchInvokeUrl + ") :" + "(耗时)" + timeCost + ": (数量)" + list.size());
        // 暂停几秒 防止被限制
        DouBanSpider.friendlyToDouban(5);
        return list;
    }

    public String getCatchUrl() {
        return catchUrl;
    }
}
